#!/usr/bin/env python
#
# url-extractor.py
#

__version = "0.1"

def url_extract_file(base, filename):
    """ extract urls from file """

    import re
    import sys
    from urlparse import urlparse, urlunparse

    base_params = urlparse(base)
    if base_params.scheme == '' or base_params.netloc == '':
        print "Incorrect base parameter: %s" % base
        sys.exit(1)

    url_selectors = (
        "(?mi)((?:http|https|ftp)://[a-z0-9\.\:/\~\%\-\?\=#_&]+)",
        "(?mi)href=\"([a-z0-9\.\:/\~\%\-\?\=#_&]+)\"",
        "(?mi)src=\"([a-z0-9\.\:/\~\%\-\?\=#_&]+)\"",
    )

    data = open(filename, "r").read()
    for url_selector in url_selectors:
        urls = re.findall(url_selector, data)
        for url in set(urls):
            (scheme, netloc, path, params, query, fragment) = urlparse(url)
            if scheme and netloc:
                print url
            else:
                print urlunparse([base_params.scheme, base_params.netloc, path, \
                                    params, query, fragment])


if __name__ == "__main__":
    from optparse import OptionParser
    
    parser = OptionParser(version="%prog, v." + __version)
    parser.add_option("-f", "--file", dest="file", help="saved html or rss file")
    parser.add_option("-b", "--base", dest="base", help="url base, for example: http://localhost/")
    (options, args) = parser.parse_args()
    
    if (options.file == None) or (options.base == None):
        parser.error("Incorrect number of arguments, \n\nuse -h or --help for detail\n")

    url_extract_file(options.base, options.file)
